{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pynvml as N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _decode(b):\n",
    "            if isinstance(b, bytes):\n",
    "                return b.decode('utf-8')    # for python3, to unicode\n",
    "            return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "N.nvmlInit()\n",
    "handle = N.nvmlDeviceGetHandleByIndex(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pynvml.LP_struct_c_nvmlDevice_t at 0x7f5c84130cc0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = _decode(N.nvmlDeviceGetName(handle))\n",
    "uuid = _decode(N.nvmlDeviceGetUUID(handle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Tesla P100-PCIE-12GB'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'GPU-b2dd4fad-98f0-c52a-bf2f-8bacdfade52f'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uuid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "memory = N.nvmlDeviceGetMemoryInfo(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pynvml.c_nvmlMemory_t at 0x7f5c8412a0c0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'string' has no attribute 'join'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-01616cf241d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pynvml.py\u001b[0m in \u001b[0;36m__str__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    397\u001b[0m                 \u001b[0mfmt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fmt_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"<default>\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    398\u001b[0m             \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfmt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 399\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"(\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\", \"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\")\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mc_nvmlUnitInfo_t\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_PrintableStructure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'string' has no attribute 'join'"
     ]
    }
   ],
   "source": [
    "print(memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12099.5"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "memory.used/(1024*1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from core import GPUStatCollection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_stats = GPUStatCollection.new_query()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPUStatCollection(host=p100, [\n",
       "  [0] Tesla P100-PCIE-12GB | 58°C,  99 % | 12099 / 12198 MB | gsx(4111M) gsx(3989M) gsx(3989M)\n",
       "  [1] Tesla P100-PCIE-12GB | 61°C,  99 % | 11512 / 12198 MB | gsx(2467M) gsx(2467M) gsx(2467M) gsx(4101M)\n",
       "  [2] Tesla P100-PCIE-12GB | 79°C,  94 % | 11793 / 12198 MB | mzy(11779M)\n",
       "  [3] Tesla P100-PCIE-12GB | 67°C, 100 % |  9858 / 12198 MB | pfren(2463M) gsx(2457M) pfren(2465M) pfren(2463M)\n",
       "])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12099"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_stats.gpus[0].memory_used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPUStatCollection(host=p100, [\n",
       "  [0] Tesla P100-PCIE-12GB | 58°C,  99 % | 12099 / 12198 MB | gsx(4111M) gsx(3989M) gsx(3989M)\n",
       "  [1] Tesla P100-PCIE-12GB | 61°C,  99 % | 11512 / 12198 MB | gsx(2467M) gsx(2467M) gsx(2467M) gsx(4101M)\n",
       "  [2] Tesla P100-PCIE-12GB | 79°C,  94 % | 11793 / 12198 MB | mzy(11779M)\n",
       "  [3] Tesla P100-PCIE-12GB | 67°C, 100 % |  9858 / 12198 MB | pfren(2463M) gsx(2457M) pfren(2465M) pfren(2463M)\n",
       "])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = ['0','1','2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.remove('1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0', '2']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "sckey = 'SCU117143T6c8ab81691bec379fc7edfbbac32526c5f7eb71836b34'\n",
    "empty_card = 0\n",
    "s = f'gpu stats: \\n\\n'\n",
    "for i in gpu_stats.gpus:\n",
    "    s+= f'gpu {i.index}: | memory used {i.memory_used}/{i.memory_total} | users: '\n",
    "    for p in i.processes:\n",
    "        username = p['username']\n",
    "        mem = p['gpu_memory_usage']\n",
    "        s+= f'{username} use {mem}|'\n",
    "    s+= '\\n\\n'\n",
    "    \n",
    "sc_res_raw = requests.post(\n",
    "            f'https://sc.ftqq.com/{sckey}.send',\n",
    "            data={\n",
    "                'text': f'空闲的GPU是{empty_card}',\n",
    "                'desp': f'{s}',\n",
    "            },\n",
    "            timeout=1,)\n",
    "print(tsc_res_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"errno\":0,\"errmsg\":\"success\",\"dataset\":\"done\"}\n"
     ]
    }
   ],
   "source": [
    "print(sc_res_raw.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from core import GPUStatCollection\n",
    "import json\n",
    "\n",
    "sckey = 'SCU117143T6c8ab81691bec379fc7edfbbac32526c5f7eb71836b34'\n",
    "empty_card = []\n",
    "flag = True # 避免重复发送gpu推送\n",
    "\n",
    "\n",
    "gpu_stats = GPUStatCollection.new_query()\n",
    "for i in gpu_stats.gpus:\n",
    "    if i.memory_used < 100:\n",
    "        if i.index not in empty_card:\n",
    "            empty_card.append(i.index)\n",
    "            flag = True\n",
    "    else:\n",
    "        if i.index in empty_card:\n",
    "            empty_card.remove(i.index)\n",
    "            \n",
    "\n",
    "if (empty_card is not None) and (flag):\n",
    "    push_to_wechat()\n",
    "    flag = False\n",
    "\n",
    "\n",
    "def push_to_wechat():\n",
    "    \n",
    "    sc_res_raw = requests.post(\n",
    "            f'https://sc.ftqq.com/{sckey}.send',\n",
    "            data={\n",
    "                'text': f'空闲的GPU是{empty_card}',\n",
    "                'desp': f'{gpu_stats}',\n",
    "            },\n",
    "            timeout=1,\n",
    "        )\n",
    "    try:\n",
    "        return_json = json.loads(sc_res_raw)\n",
    "    except:\n",
    "        raise RuntimeError(f'Server 酱的返回值不能解析为 JSON，可能您的 SCKEY 配置有误'\n",
    "                          f'API 的返回是：\\n{sc_res_raw}\\n您输入的 SCKEY 为\\n{sckey}')\n",
    "    return_json = sc_res.get('errno')\n",
    "        if errno != 0:\n",
    "            raise RuntimeError(\n",
    "                f'Server 酱调用失败，可能您的 SCKEY 配置有误。API 的返回是：\\n{sc_res_raw}\\n'\n",
    "                f'您输入的 SCKEY 为\\n{sckey}')\n",
    "\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
